#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: copy_crawl_data_new.py
@time: 2017/12/27 22:02
"""
import sys

sys.path.append('..')

from common.mongo import MongDb
from logger import Logger

MONGO_DB_SOURCE = {
    'host': '172.16.215.2',
    'port': 40042,
    'db': 'crawl_data_new',
    'username': 'work',
    'password': 'haizhi'
}

MONGO_DB_TARGET = {
    'host': '172.16.215.2',
    'port': 40042,
    'db': 'mfile',
    'username': 'work',
    'password': 'haizhi'
}

# 日志模块
log = Logger('copy_crawl_data_new.log').get_logger()

source_db = MongDb(MONGO_DB_SOURCE['host'], MONGO_DB_SOURCE['port'], MONGO_DB_SOURCE['db'],
                   MONGO_DB_SOURCE['username'],
                   MONGO_DB_SOURCE['password'], log=log)

target_db = MongDb(MONGO_DB_TARGET['host'], MONGO_DB_TARGET['port'], MONGO_DB_TARGET['db'],
                   MONGO_DB_TARGET['username'],
                   MONGO_DB_TARGET['password'], log=log)


def main():
    source_table_list = [
        'new_gsxt_search_page',
        'new_zhejiang_search_page',
        'old_zhejiang_search_page',
    ]

    total_count = 0
    for table_name in source_table_list:
        result_list = []
        count = 0
        for item in source_db.traverse_batch(table_name, {}, 300):
            result_list.append(item)
            count += 1
            total_count += 1

            if len(result_list) >= 300:
                target_db.insert_batch_data(table_name, result_list)
                del result_list[:]

            if count % 1000 == 0:
                log.info("当前进度: table = {} count = {} total = {}".format(
                    table_name, count, total_count))

        if len(result_list) > 0:
            target_db.insert_batch_data(table_name, result_list)
            del result_list[:]

        # 删除表
        source_db.drop(table_name)
        log.info("当前表复制完成: table = {}".format(table_name))
    log.info("总共复制数据量: total = {}".format(total_count))


if __name__ == '__main__':
    main()
